<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta name="referrer" content="no-referrer"><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>18.Python爬虫-requests模块 | 王扣笔记</title><meta name="author" content="王扣"><meta name="copyright" content="王扣"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="Python爬虫 - 03.requests模块进阶1.多线程12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061&quot;&quot;&quot;多线程异步&quot;&quot;&quot;&quot;&quot;&amp;qu">
<meta property="og:type" content="article">
<meta property="og:title" content="18.Python爬虫-requests模块">
<meta property="og:url" content="http://grace.666443.xyz/posts/ae1a0acd.html">
<meta property="og:site_name" content="王扣笔记">
<meta property="og:description" content="Python爬虫 - 03.requests模块进阶1.多线程12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061&quot;&quot;&quot;多线程异步&quot;&quot;&quot;&quot;&quot;&amp;qu">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://p.ananas.chaoxing.com/star3/origin/b230c5f9060b1f422a86e332d8e37926.png">
<meta property="article:published_time" content="2022-12-08T12:53:30.000Z">
<meta property="article:modified_time" content="2022-12-08T12:58:26.978Z">
<meta property="article:author" content="王扣">
<meta property="article:tag" content="笔记 开发">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://p.ananas.chaoxing.com/star3/origin/b230c5f9060b1f422a86e332d8e37926.png"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="http://grace.666443.xyz/posts/ae1a0acd"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"/search.xml","preload":true,"languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#1f1f1f","position":"bottom-left"},
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '18.Python爬虫-requests模块',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2022-12-08 20:58:26'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.2"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="https://i.loli.net/2021/02/24/5O1day2nriDzjSu.png" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">22</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">1</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fa fa-heartbeat"></i><span> 清单</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li><li><a class="site-page child" href="/photos/"><i class="fa-fw fas fa-images"></i><span> 照片</span></a></li><li><a class="site-page child" href="/movies/"><i class="fa-fw fas fa-video"></i><span> 电影</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('https://p.ananas.chaoxing.com/star3/origin/b230c5f9060b1f422a86e332d8e37926.png')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">王扣笔记</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fa fa-heartbeat"></i><span> 清单</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li><li><a class="site-page child" href="/photos/"><i class="fa-fw fas fa-images"></i><span> 照片</span></a></li><li><a class="site-page child" href="/movies/"><i class="fa-fw fas fa-video"></i><span> 电影</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">18.Python爬虫-requests模块</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2022-12-08T12:53:30.000Z" title="发表于 2022-12-08 20:53:30">2022-12-08</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2022-12-08T12:58:26.978Z" title="更新于 2022-12-08 20:58:26">2022-12-08</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/Python%E7%88%AC%E8%99%AB/">Python爬虫</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="18.Python爬虫-requests模块"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="Python爬虫-03-requests模块进阶"><a href="#Python爬虫-03-requests模块进阶" class="headerlink" title="Python爬虫 - 03.requests模块进阶"></a>Python爬虫 - 03.requests模块进阶</h1><h2 id="1-多线程"><a href="#1-多线程" class="headerlink" title="1.多线程"></a><strong>1.多线程</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">多线程</span></span><br><span class="line"><span class="string">异步</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">进程：资源单位 ( 公司 )。每一个进程至少要有一个线程</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">线程：执行单位 (员工干活)</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 启动每一个程序默认都会有一个主线程</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># def func():</span></span><br><span class="line"><span class="comment">#     for i in range(1000):</span></span><br><span class="line"><span class="comment">#         print(&quot;func&quot;, i)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># if __name__ == &#x27;__main__&#x27;:</span></span><br><span class="line"><span class="comment">#     print(&quot;您好呀&quot;)</span></span><br><span class="line"><span class="comment">#     func()</span></span><br><span class="line"><span class="comment">#     for i in range(1000):</span></span><br><span class="line"><span class="comment">#         print(&quot;main&quot;, i)</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># # 多线程 第一套写法</span></span><br><span class="line"><span class="comment"># from threading import Thread  # 导入线程类</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># def func():</span></span><br><span class="line"><span class="comment">#     for i in range(1000):</span></span><br><span class="line"><span class="comment">#         print(&quot;func&quot;, i)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># if __name__ == &#x27;__main__&#x27;:</span></span><br><span class="line"><span class="comment">#     t = Thread(target=func)  # 线程分配目标任务</span></span><br><span class="line"><span class="comment">#     t.start()  # 开始执行该线程。多线程状态为可以工作状态，具体的执行时间由CPU决定</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#     for i in range(1000):</span></span><br><span class="line"><span class="comment">#         print(&quot;main&quot;, i)</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 多线程 第2套写法 ,用的比较多</span></span><br><span class="line"><span class="keyword">from</span> threading <span class="keyword">import</span> Thread  <span class="comment"># 导入线程类</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyThread</span>(<span class="title class_ inherited__">Thread</span>):  <span class="comment"># 类 继承</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">run</span>(<span class="params">self</span>):  <span class="comment"># 固定的  -&gt; 当线程被执行的时候，被执行的就是run()</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1000</span>):</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;子线程&quot;</span>, i)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line"></span><br><span class="line">    t = MyThread()</span><br><span class="line">    t.start()</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1000</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;主线程&quot;</span>, i)</span><br><span class="line"></span><br></pre></td></tr></table></figure>


<h2 id="2-多进程"><a href="#2-多进程" class="headerlink" title="2.多进程"></a><strong>2.多进程</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">多进程比较耗费资源，不是很建议</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="comment"># 导包</span></span><br><span class="line"><span class="keyword">from</span> multiprocessing <span class="keyword">import</span> Process</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func</span>():</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1000</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;子进程&quot;</span>, i)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    p = Process(target=func)</span><br><span class="line">    p.start()</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1000</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;主进程&quot;</span>, i)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="3-线程池"><a href="#3-线程池" class="headerlink" title="3.线程池"></a><strong>3.线程池</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string"># 线程池 ：一次性开辟一些线程，我们用户直接给线程池子提交任务。线程任务的调度交给线程池来完成</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ThreadPoolExecutor, ProcessPoolExecutor</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">fn</span>(<span class="params">name</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1000</span>):</span><br><span class="line">        <span class="built_in">print</span>(name, i)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    <span class="comment"># 创建线程池</span></span><br><span class="line">    <span class="keyword">with</span> ThreadPoolExecutor(<span class="number">50</span>) <span class="keyword">as</span> t:  <span class="comment"># 创建一个有50个线程组成的线程池</span></span><br><span class="line">        <span class="comment"># 准备多少个任务</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100</span>):</span><br><span class="line">            t.submit(fn, name=<span class="string">f&quot;线程<span class="subst">&#123;i&#125;</span>&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 等待线程池中的任务全部执行完毕，才继续执行( 守护 )</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;123&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="4-线程池实战"><a href="#4-线程池实战" class="headerlink" title="4.线程池实战"></a><strong>4.线程池实战</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string"># 1.如何提取单个页面的数据</span></span><br><span class="line"><span class="string"># 2.上线程池，多个页面同时抓取</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ThreadPoolExecutor</span><br><span class="line"></span><br><span class="line">f = <span class="built_in">open</span>(<span class="string">&quot;data.csv&quot;</span>, mode=<span class="string">&quot;w&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">csvWrite = csv.writer(f)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">download_one_page</span>(<span class="params">url</span>):</span><br><span class="line">    <span class="comment"># 拿到页面源代码</span></span><br><span class="line">    resp = requests.get(url)</span><br><span class="line">    html = etree.HTML(resp.text)</span><br><span class="line">    table = html.xpath(<span class="string">&quot;/html/body/div[2]/div[4]/div[1]/table&quot;</span>)[<span class="number">0</span>]</span><br><span class="line">    trs = table.xpath(<span class="string">&quot;./tr[position()&gt;1])&quot;</span>)</span><br><span class="line">    <span class="comment"># 拿到每个tr</span></span><br><span class="line">    <span class="keyword">for</span> tr <span class="keyword">in</span> trs:</span><br><span class="line">        tr.xpath(<span class="string">&quot;./td/text()&quot;</span>)</span><br><span class="line">        <span class="comment"># 对数据做一个简单的处理：\\ /去掉  生成器</span></span><br><span class="line">        txt = (item.replace(<span class="string">&quot;\\&quot;</span>, <span class="string">&#x27;&#x27;</span>).replace(<span class="string">&quot;/&quot;</span>, <span class="string">&quot;&quot;</span>) <span class="keyword">for</span> item <span class="keyword">in</span> txt)</span><br><span class="line">        <span class="comment"># 写数据到文件中</span></span><br><span class="line">        csvWrite.writerow(txt)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;url提取完毕&quot;</span>)</span><br><span class="line">    f.close()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    <span class="keyword">with</span> ThreadPoolExecutor(<span class="number">50</span>) <span class="keyword">as</span> t:  <span class="comment"># 上50个线程</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">200</span>):  <span class="comment"># 循环多少页</span></span><br><span class="line">            t.submit(download_one_page, <span class="string">f&quot;url链接带有页面的&quot;</span>)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;全部下载完毕!&quot;</span>)</span><br><span class="line">    <span class="keyword">pass</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="5-多线程代理ip"><a href="#5-多线程代理ip" class="headerlink" title="5.多线程代理ip"></a><strong>5.多线程代理ip</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">通过多线程方式获取IP代理池</span></span><br><span class="line"><span class="string">目标： https://www.kuaidaili.com/free/inha/2/</span></span><br><span class="line"><span class="string"># 思路：</span></span><br><span class="line"><span class="string">1.提取单页面数据,并验证有效性，有效的保留</span></span><br><span class="line"><span class="string">2.上线程池，多页面同时抓取</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ThreadPoolExecutor</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">f = <span class="built_in">open</span>(<span class="string">&quot;ip代理.csv&quot;</span>, mode=<span class="string">&#x27;w&#x27;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">csw_write = csv.writer(f)</span><br><span class="line">csw_write.writerow([<span class="string">&quot;有效IP代理&quot;</span>])</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">down_onepage_ip</span>(<span class="params">url</span>):</span><br><span class="line">    resp = requests.get(url, headers=headers, verify=<span class="literal">False</span>)</span><br><span class="line">    html = etree.HTML(resp.text)</span><br><span class="line">    trs = html.xpath(<span class="string">&quot;/html/body/div[2]/section[1]/div/div/div/div[1]/div/article/div[2]/div/figure/table/tbody/tr&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> tr <span class="keyword">in</span> trs:</span><br><span class="line">        ipList = tr.xpath(<span class="string">&quot;./td/text()&quot;</span>)</span><br><span class="line">        ip = ipList[<span class="number">0</span>]</span><br><span class="line">        port = ipList[<span class="number">1</span>]</span><br><span class="line">        httpType = <span class="built_in">str</span>.lower(ipList[<span class="number">4</span>])</span><br><span class="line">        httpPorx = <span class="string">f&#x27;<span class="subst">&#123;httpType&#125;</span>://<span class="subst">&#123;ip&#125;</span>:<span class="subst">&#123;port&#125;</span>&#x27;</span></span><br><span class="line">        check_proxie_url = <span class="string">&quot;http://httpbin.org/ip&quot;</span></span><br><span class="line">        proxie = &#123;</span><br><span class="line">            httpType: httpPorx</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            resprox = requests.get(check_proxie_url, headers=headers, proxies=proxie, timeout=<span class="number">10</span>)</span><br><span class="line">            csw_write.writerow([httpPorx])</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">f&quot;==&gt; 代理可用，IP为\t <span class="subst">&#123;httpPorx&#125;</span>&quot;</span>)</span><br><span class="line">        <span class="keyword">except</span> Exception:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">f&quot;请求失败，代理IP无效！ &quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;===&gt; 当前 <span class="subst">&#123;url.split(<span class="string">&#x27;/&#x27;</span>)[-<span class="number">1</span>]&#125;</span>页 IP页面数据提取完毕 &quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    <span class="comment"># down_onepage_ip(&quot;https://www.beesproxy.com/free/page/1&quot;)</span></span><br><span class="line">    <span class="comment"># check_ip(&quot;http://www.baidu.com&quot;)</span></span><br><span class="line">    <span class="comment"># 创建线程池里面有多少个线程</span></span><br><span class="line">    <span class="keyword">with</span> ThreadPoolExecutor(<span class="number">25</span>) <span class="keyword">as</span> t:</span><br><span class="line">        <span class="comment"># 多少个页面循环</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">20</span>):</span><br><span class="line">            t.submit(down_onepage_ip, <span class="string">f&quot;https://www.beesproxy.com/free/page/<span class="subst">&#123;i&#125;</span>&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;===&gt; 全部提取完毕&quot;</span>)</span><br><span class="line">    f.close()</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="6-协程"><a href="#6-协程" class="headerlink" title="6.协程"></a><strong>6.协程</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">协程</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;我爱黎明&quot;</span>)</span><br><span class="line">    time.sleep(<span class="number">3</span>)  <span class="comment"># 让当前的线程处于阻塞状态，CPU是不为我工作的</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;我真的爱黎明&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    func()</span><br><span class="line"></span><br><span class="line"><span class="comment"># input() 程序也是处于阻塞状态</span></span><br><span class="line"><span class="comment"># requestes.get(url) 在网络请求返回数据之前，程序也是处于阻塞状态的</span></span><br><span class="line"><span class="comment"># 一般情况下，当程序处理IO操作时候，线程都会处于阻塞状态</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 协程：当程序遇见了IO操作时候，可以选择性的切换到其他任务上</span></span><br><span class="line"><span class="comment"># 在微观上 是一个任务一个任务的进行切换，切换条件一般就是IO操作</span></span><br><span class="line"><span class="comment"># 在宏观上，我们能看到的其实是多个任务一起在执行</span></span><br><span class="line"><span class="comment"># 多任务异步操作</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 上方所讲的一切，都是在单线程的条件下</span></span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="7-python编写协程程序"><a href="#7-python编写协程程序" class="headerlink" title="7.python编写协程程序"></a><strong>7.python编写协程程序</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">python编写协程的程序</span></span><br><span class="line"><span class="string">市面上有四种方式，这里只学习一种比较常见的方式</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"></span><br><span class="line"><span class="comment"># # 异步</span></span><br><span class="line"><span class="comment"># async def func():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚&quot;)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># if __name__ == &#x27;__main__&#x27;:</span></span><br><span class="line"><span class="comment">#     g = func()  # 此时的函数是异步协程函数，此时函数执行得到的是一个协程对象</span></span><br><span class="line"><span class="comment">#     asyncio.run(g)  # 协程程序运行需要asyncio模块的支持</span></span><br><span class="line"><span class="comment">#     # print(g)</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># async def func1():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚&quot;)</span></span><br><span class="line"><span class="comment">#     # time.sleep(3)  # 当程序出现了同步操作的时候，异步就中断了</span></span><br><span class="line"><span class="comment">#     await asyncio.sleep(3)  # 异步操作的代码</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚&quot;)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># async def func2():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚2&quot;)</span></span><br><span class="line"><span class="comment">#     # time.sleep(3)</span></span><br><span class="line"><span class="comment">#     await asyncio.sleep(3)  # 异步操作的代码</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚2&quot;)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># async def func3():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚3&quot;)</span></span><br><span class="line"><span class="comment">#     # time.sleep(3)</span></span><br><span class="line"><span class="comment">#     await asyncio.sleep(3)  # 异步操作的代码 ，await 挂起</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚4&quot;)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># if __name__ == &#x27;__main__&#x27;:</span></span><br><span class="line"><span class="comment">#     f1 = func1()</span></span><br><span class="line"><span class="comment">#     f2 = func2()</span></span><br><span class="line"><span class="comment">#     f3 = func3()</span></span><br><span class="line"><span class="comment">#     tasks = [</span></span><br><span class="line"><span class="comment">#         f1, f2, f3</span></span><br><span class="line"><span class="comment">#     ]</span></span><br><span class="line"><span class="comment">#     t1 = time.time()</span></span><br><span class="line"><span class="comment">#     # 一次性启动多个任务（协程）</span></span><br><span class="line"><span class="comment">#     asyncio.run(asyncio.wait(tasks))</span></span><br><span class="line"><span class="comment">#     t2 = time.time()</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># print(t2 - t1)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># =====================下面是 官方推荐操作方式</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># async def func1():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚&quot;)</span></span><br><span class="line"><span class="comment">#     await asyncio.sleep(3)  # 异步操作的代码</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚&quot;)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># async def func2():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚2&quot;)</span></span><br><span class="line"><span class="comment">#     await asyncio.sleep(3)  # 异步操作的代码</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚2&quot;)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># async def func3():</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚3&quot;)</span></span><br><span class="line"><span class="comment">#     await asyncio.sleep(3)  # 异步操作的代码 ，await 挂起</span></span><br><span class="line"><span class="comment">#     print(&quot;你好呀，我叫赛利亚4&quot;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># async def main():</span></span><br><span class="line"><span class="comment">#     # 第一种写法 不是很推荐</span></span><br><span class="line"><span class="comment">#     # f1 = func1()</span></span><br><span class="line"><span class="comment">#     # await f1 # 一般await挂起操作放在协程对象前面</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#     # 第二种写法 （推荐）</span></span><br><span class="line"><span class="comment">#     tasks = [</span></span><br><span class="line"><span class="comment">#         asyncio.create_task(func1()), # 创建一个协程任务 py3.8以后用这种方法</span></span><br><span class="line"><span class="comment">#         asyncio.create_task(func2()),</span></span><br><span class="line"><span class="comment">#         asyncio.create_task(func3())</span></span><br><span class="line"><span class="comment">#      ]</span></span><br><span class="line"><span class="comment">#     await asyncio.wait(tasks)</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># if __name__ == &#x27;__main__&#x27;:</span></span><br><span class="line"><span class="comment">#     asyncio.run(main())</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># ============协程模拟爬虫下载视频 =====</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在爬虫领域的应用</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">def</span> <span class="title function_">download</span>(<span class="params">url</span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;准备下载...&quot;</span>)</span><br><span class="line">    <span class="keyword">await</span> asyncio.sleep(<span class="number">2</span>)  <span class="comment"># 模拟的是网络请求， requests.get()这个是同步操作 要注意</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;下载完成&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">def</span> <span class="title function_">main</span>():</span><br><span class="line">    urls = [</span><br><span class="line">        <span class="string">&quot;http://www.baidu.com&quot;</span>,</span><br><span class="line">        <span class="string">&quot;http://bilibili.com&quot;</span>,</span><br><span class="line">        <span class="string">&quot;http://www.163.com&quot;</span></span><br><span class="line">    ]</span><br><span class="line"></span><br><span class="line">    tasks = []</span><br><span class="line">    <span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">        d = asyncio.create_task(download(url))  <span class="comment"># 创建协程对象</span></span><br><span class="line">        tasks.append(d)  <span class="comment"># 将协程对象 添加在创建的列表任务里面</span></span><br><span class="line">    <span class="keyword">await</span> asyncio.wait(tasks)  <span class="comment"># 将任务列表 一起去跑</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># tasks = [asyncio.create_task(download(url)) for url in urls]  # 这么干也是可以的</span></span><br><span class="line">    </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    asyncio.run(main())  <span class="comment"># 运行协程</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="8-协程-异步-aiohttp模块"><a href="#8-协程-异步-aiohttp模块" class="headerlink" title="8.协程(异步)aiohttp模块"></a><strong>8.协程(异步)aiohttp模块</strong></h2><blockquote>
<p>**当前python的环境是 python 3.10</p>
</blockquote>
<figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">异步的 http操作</span></span><br><span class="line"><span class="string">requests.get() 是同步的代码  ==&gt; 异步操作的 需要用 aiohttp异步</span></span><br><span class="line"><span class="string">目标网站</span></span><br><span class="line"><span class="string">http://umei.cc</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 导包 异步操作模块</span></span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="comment"># 导包 异步http请求操作模块</span></span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"></span><br><span class="line">urls = [</span><br><span class="line">    <span class="comment"># https 会报错，暂时没学到解决办法</span></span><br><span class="line">    <span class="string">&quot;http://img.netbian.com/file/2022/0805/233217zUWmF.jpg&quot;</span>,</span><br><span class="line">    <span class="string">&quot;http://img.netbian.com/file/2022/1031/234333kApFn.jpg&quot;</span>,</span><br><span class="line">    <span class="string">&quot;http://img.netbian.com/file/2022/0127/235536d7bRL.jpg&quot;</span></span><br><span class="line">]</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 异步协程 下载三张图片</span></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">def</span> <span class="title function_">aiodownload</span>(<span class="params">url</span>):</span><br><span class="line">    name = url.rsplit(<span class="string">&#x27;/&#x27;</span>, <span class="number">1</span>)[<span class="number">1</span>]  <span class="comment"># 从右侧分割</span></span><br><span class="line">    <span class="comment"># 发送请求</span></span><br><span class="line">    <span class="comment"># 得到图片内容</span></span><br><span class="line">    <span class="comment"># 保存到文件</span></span><br><span class="line">    <span class="comment"># s = aiohttp.ClientSession()   # 等价于 requests模块</span></span><br><span class="line">    <span class="comment"># with 相当于文件操作，会自动close</span></span><br><span class="line">    <span class="keyword">async</span> <span class="keyword">with</span> aiohttp.ClientSession() <span class="keyword">as</span> session:</span><br><span class="line">        <span class="keyword">async</span> <span class="keyword">with</span> session.get(url) <span class="keyword">as</span> resp:</span><br><span class="line">            <span class="comment"># 请求回来了，写入文件</span></span><br><span class="line">            <span class="comment"># 可以自己去学习一个模块,aiofiles 异步模块</span></span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">&quot;异步下载的图片/&quot;</span> + name, mode=<span class="string">&#x27;wb&#x27;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(<span class="keyword">await</span> resp.content.read())  <span class="comment"># 读取内容是异步的，需要await挂起</span></span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;success!&quot;</span>, name)</span><br><span class="line">    <span class="comment"># resp.content.read() # 读取内容 等价于 resp.content</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">def</span> <span class="title function_">main</span>():</span><br><span class="line">    tasks = []</span><br><span class="line">    <span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">        tasks.append(asyncio.create_task(aiodownload(url)))</span><br><span class="line"></span><br><span class="line">    <span class="keyword">await</span> asyncio.wait(tasks)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    asyncio.run(main())</span><br><span class="line"></span><br></pre></td></tr></table></figure>






<h2 id="9-异步爬虫，扒光一部小说"><a href="#9-异步爬虫，扒光一部小说" class="headerlink" title="9.异步爬虫，扒光一部小说"></a><strong>9.异步爬虫，扒光一部小说</strong></h2><figure class="highlight py"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">异步，扒光一部小说  异步操作感觉比多线程的还要快</span></span><br><span class="line"><span class="string">目标：百度小说</span></span><br><span class="line"><span class="string">官网：https://dushu.baidu.com/pc/detail?gid=4306063500</span></span><br><span class="line"><span class="string">URL编码解码在线工具 https://www.bejson.com/enc/urlencode/</span></span><br><span class="line"><span class="string">书名：西游记</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string"># 获取所有章节的API</span></span><br><span class="line"><span class="string"># https://dushu.baidu.com/api/pc/getCatalog?data=&#123;&quot;book_id&quot;:&quot;4306063500&quot;&#125;</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string"># 章节内容api</span></span><br><span class="line"><span class="string"># https://dushu.baidu.com/api/pc/getChapterContent?data=&#123;&quot;book_id&quot;:&quot;4306063500&quot;,&quot;cid&quot;:&quot;4306063500|1569782250&quot;,&quot;need_bookinfo&quot;:1&#125;</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> aiofiles</span><br><span class="line"></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">1.同步操作： 访问 getCatalog 拿到所有章节的cid 和 名称</span></span><br><span class="line"><span class="string">2.异步操作： 访问getChapterContent 下载所有的章节内容</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">def</span> <span class="title function_">aiodownload</span>(<span class="params">cid, b_id, title</span>):</span><br><span class="line">    data = &#123;</span><br><span class="line">        <span class="string">&quot;book_id&quot;</span>: b_id,</span><br><span class="line">        <span class="string">&quot;cid&quot;</span>: <span class="string">f&quot;<span class="subst">&#123;b_id&#125;</span>|<span class="subst">&#123;cid&#125;</span>&quot;</span>,</span><br><span class="line">        <span class="string">&quot;need_bookinfo&quot;</span>: <span class="number">1</span></span><br><span class="line">    &#125;</span><br><span class="line">    data = json.dumps(data)  <span class="comment"># 转换为json</span></span><br><span class="line">    url = <span class="string">f&quot;https://dushu.baidu.com/api/pc/getChapterContent?data=<span class="subst">&#123;data&#125;</span>&quot;</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 异步请求</span></span><br><span class="line">    <span class="keyword">async</span> <span class="keyword">with</span> aiohttp.ClientSession() <span class="keyword">as</span> session:</span><br><span class="line">        <span class="keyword">async</span> <span class="keyword">with</span> session.get(url, ssl=<span class="literal">False</span>) <span class="keyword">as</span> resp:</span><br><span class="line">            dic = <span class="keyword">await</span> resp.json()</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 异步写到文件</span></span><br><span class="line">            <span class="keyword">try</span>:</span><br><span class="line">                <span class="keyword">async</span> <span class="keyword">with</span> aiofiles.<span class="built_in">open</span>(<span class="string">&quot;小说/&quot;</span> + title+<span class="string">&#x27;.txt&#x27;</span>, mode=<span class="string">&#x27;w&#x27;</span>, encoding=<span class="string">&#x27;utf-8&#x27;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                    <span class="keyword">await</span> f.write(dic[<span class="string">&#x27;data&#x27;</span>][<span class="string">&#x27;novel&#x27;</span>][<span class="string">&#x27;content&#x27;</span>])  <span class="comment"># 把小说内容写出</span></span><br><span class="line">            <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">                <span class="built_in">print</span>(<span class="string">f&quot;<span class="subst">&#123;title&#125;</span> error!&quot;</span>)</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="built_in">print</span>(<span class="string">f&quot;<span class="subst">&#123;title&#125;</span> success! &quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">def</span> <span class="title function_">getCatalog</span>(<span class="params">url</span>):</span><br><span class="line">    resp = requests.get(url)</span><br><span class="line">    dic = resp.json()</span><br><span class="line">    tasks = []</span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> dic[<span class="string">&#x27;data&#x27;</span>][<span class="string">&#x27;novel&#x27;</span>][<span class="string">&#x27;items&#x27;</span>]:  <span class="comment"># item就是对应的每一个章节的名称和cid</span></span><br><span class="line">        title = item[<span class="string">&#x27;title&#x27;</span>]</span><br><span class="line">        cid = item[<span class="string">&#x27;cid&#x27;</span>]</span><br><span class="line">        <span class="built_in">print</span>(cid, title)</span><br><span class="line">        <span class="comment"># 准备异步任务</span></span><br><span class="line">        tasks.append(asyncio.create_task(aiodownload(cid, b_id, title)))</span><br><span class="line"></span><br><span class="line">    <span class="keyword">await</span> asyncio.wait(tasks)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    b_id = <span class="string">&quot;4306063500&quot;</span></span><br><span class="line">    url = <span class="string">&#x27;https://dushu.baidu.com/api/pc/getCatalog?data=&#123;&quot;book_id&quot;:&quot;&#x27;</span> + b_id + <span class="string">&#x27;&quot;&#125;&#x27;</span></span><br><span class="line">    asyncio.run(getCatalog(url))</span><br><span class="line">   <span class="comment"># asyncio.get_event_loop().run_until_complete(getCatalog(url))</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="10-综合案例-抓取一部视频"><a href="#10-综合案例-抓取一部视频" class="headerlink" title="10.综合案例 抓取一部视频"></a><strong>10.综合案例 抓取一部视频</strong></h2></article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="http://grace.666443.xyz">王扣</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="http://grace.666443.xyz/posts/ae1a0acd.html">http://grace.666443.xyz/posts/ae1a0acd.html</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="http://grace.666443.xyz" target="_blank">王扣笔记</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"></div><div class="post_share"><div class="social-share" data-image="https://p.ananas.chaoxing.com/star3/origin/b230c5f9060b1f422a86e332d8e37926.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="next-post pull-full"><a href="/posts/ed7a56b2.html"><img class="next-cover" src="https://p.ananas.chaoxing.com/star3/origin/8f2179d04bfe0e48cc706ae112c97952.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">17.Python爬虫-数据解析与提取</div></div></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="https://i.loli.net/2021/02/24/5O1day2nriDzjSu.png" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">王扣</div><div class="author-info__description">常用开发中的笔记记录</div></div><div class="card-info-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">22</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">1</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">This is my Blog 技术笔记</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#Python%E7%88%AC%E8%99%AB-03-requests%E6%A8%A1%E5%9D%97%E8%BF%9B%E9%98%B6"><span class="toc-number">1.</span> <span class="toc-text">Python爬虫 - 03.requests模块进阶</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#1-%E5%A4%9A%E7%BA%BF%E7%A8%8B"><span class="toc-number">1.1.</span> <span class="toc-text">1.多线程</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-%E5%A4%9A%E8%BF%9B%E7%A8%8B"><span class="toc-number">1.2.</span> <span class="toc-text">2.多进程</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3-%E7%BA%BF%E7%A8%8B%E6%B1%A0"><span class="toc-number">1.3.</span> <span class="toc-text">3.线程池</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-%E7%BA%BF%E7%A8%8B%E6%B1%A0%E5%AE%9E%E6%88%98"><span class="toc-number">1.4.</span> <span class="toc-text">4.线程池实战</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-%E5%A4%9A%E7%BA%BF%E7%A8%8B%E4%BB%A3%E7%90%86ip"><span class="toc-number">1.5.</span> <span class="toc-text">5.多线程代理ip</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#6-%E5%8D%8F%E7%A8%8B"><span class="toc-number">1.6.</span> <span class="toc-text">6.协程</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#7-python%E7%BC%96%E5%86%99%E5%8D%8F%E7%A8%8B%E7%A8%8B%E5%BA%8F"><span class="toc-number">1.7.</span> <span class="toc-text">7.python编写协程程序</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#8-%E5%8D%8F%E7%A8%8B-%E5%BC%82%E6%AD%A5-aiohttp%E6%A8%A1%E5%9D%97"><span class="toc-number">1.8.</span> <span class="toc-text">8.协程(异步)aiohttp模块</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#9-%E5%BC%82%E6%AD%A5%E7%88%AC%E8%99%AB%EF%BC%8C%E6%89%92%E5%85%89%E4%B8%80%E9%83%A8%E5%B0%8F%E8%AF%B4"><span class="toc-number">1.9.</span> <span class="toc-text">9.异步爬虫，扒光一部小说</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#10-%E7%BB%BC%E5%90%88%E6%A1%88%E4%BE%8B-%E6%8A%93%E5%8F%96%E4%B8%80%E9%83%A8%E8%A7%86%E9%A2%91"><span class="toc-number">1.10.</span> <span class="toc-text">10.综合案例 抓取一部视频</span></a></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/posts/ae1a0acd.html" title="18.Python爬虫-requests模块"><img src="https://p.ananas.chaoxing.com/star3/origin/b230c5f9060b1f422a86e332d8e37926.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="18.Python爬虫-requests模块"/></a><div class="content"><a class="title" href="/posts/ae1a0acd.html" title="18.Python爬虫-requests模块">18.Python爬虫-requests模块</a><time datetime="2022-12-08T12:53:30.000Z" title="发表于 2022-12-08 20:53:30">2022-12-08</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/ed7a56b2.html" title="17.Python爬虫-数据解析与提取"><img src="https://p.ananas.chaoxing.com/star3/origin/8f2179d04bfe0e48cc706ae112c97952.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="17.Python爬虫-数据解析与提取"/></a><div class="content"><a class="title" href="/posts/ed7a56b2.html" title="17.Python爬虫-数据解析与提取">17.Python爬虫-数据解析与提取</a><time datetime="2022-12-08T12:52:55.000Z" title="发表于 2022-12-08 20:52:55">2022-12-08</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/5038ed.html" title="16.Python爬虫-基础"><img src="https://p.ananas.chaoxing.com/star3/origin/2054c03c91d338203c730610ead1b303.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="16.Python爬虫-基础"/></a><div class="content"><a class="title" href="/posts/5038ed.html" title="16.Python爬虫-基础">16.Python爬虫-基础</a><time datetime="2022-12-08T12:52:11.000Z" title="发表于 2022-12-08 20:52:11">2022-12-08</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/2b5a0751.html" title="15.Python之Mysql数据库相关"><img src="https://p.ananas.chaoxing.com/star3/origin/2a1821704670b5cf8c2289255e183d62.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="15.Python之Mysql数据库相关"/></a><div class="content"><a class="title" href="/posts/2b5a0751.html" title="15.Python之Mysql数据库相关">15.Python之Mysql数据库相关</a><time datetime="2022-12-08T12:17:02.000Z" title="发表于 2022-12-08 20:17:02">2022-12-08</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/6ebaa02b.html" title="14.Python面向对象"><img src="https://p.ananas.chaoxing.com/star3/origin/d24d9bfcd4ce3f910d6b5cdc78676d75.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="14.Python面向对象"/></a><div class="content"><a class="title" href="/posts/6ebaa02b.html" title="14.Python面向对象">14.Python面向对象</a><time datetime="2022-12-08T12:10:47.000Z" title="发表于 2022-12-08 20:10:47">2022-12-08</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2022 By 王扣</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">搜索</span><span id="loading-status"></span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="is-center" id="loading-database"><i class="fas fa-spinner fa-pulse"></i><span>  数据库加载中</span></div><div class="search-wrap"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div><hr/><div id="local-search-results"></div></div></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.min.js"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><div class="js-pjax"><script>(() => {
  const $mermaidWrap = document.querySelectorAll('#article-container .mermaid-wrap')
  if ($mermaidWrap.length) {
    window.runMermaid = () => {
      window.loadMermaid = true
      const theme = document.documentElement.getAttribute('data-theme') === 'dark' ? 'dark' : 'default'

      Array.from($mermaidWrap).forEach((item, index) => {
        const mermaidSrc = item.firstElementChild
        const mermaidThemeConfig = '%%{init:{ \'theme\':\'' + theme + '\'}}%%\n'
        const mermaidID = 'mermaid-' + index
        const mermaidDefinition = mermaidThemeConfig + mermaidSrc.textContent
        mermaid.mermaidAPI.render(mermaidID, mermaidDefinition, (svgCode) => {
          mermaidSrc.insertAdjacentHTML('afterend', svgCode)
        })
      })
    }

    const loadMermaid = () => {
      window.loadMermaid ? runMermaid() : getScript('https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.min.js').then(runMermaid)
    }

    window.pjax ? loadMermaid() : document.addEventListener('DOMContentLoaded', loadMermaid)
  }
})()</script></div><script defer="defer" id="ribbon" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/canvas-ribbon.min.js" size="150" alpha="0.6" zIndex="-1" mobile="false" data-click="false"></script><div class="fb-customerchat" id="fb-customer-chat" attribution="biz_inbox"></div><script>document.getElementById('fb-root') ? '' : document.body.insertAdjacentHTML('afterend', '<div id="fb-root"></div>')

window.fbAsyncInit = function() {
  FB.init({
    xfbml: true,
    version: 'v15.0'
  });
};

(function(d, s, id) {
  var js, fjs = d.getElementsByTagName(s)[0];
  if (d.getElementById(id)) return;
  js = d.createElement(s); js.id = id;
  js.src = 'https://connect.facebook.net/zh_CN/sdk/xfbml.customerchat.js';
  fjs.parentNode.insertBefore(js, fjs);
}(document, 'script', 'facebook-jssdk'));

if (false) {
  var chatBtnFn = () => {
    var chatBtn = document.getElementById("chat_btn")
    chatBtn.addEventListener("click", function(){
      FB.CustomerChat.show();
    });
  }
  chatBtnFn()
} else {
  if (false) {
    function chatBtnHide () {
      FB.CustomerChat.hide()
    }
    function chatBtnShow () {
      FB.CustomerChat.show(false)
    }
  }
}</script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>